import requests
from bs4 import BeautifulSoup
def get_links(url):
    response = requests.get(url)
    response.encoding = 'GBK'
    soup = BeautifulSoup(response.text, 'html.parser')
    links = soup.find_all('a')
    all_classes = set()
    for tag in soup.find_all(True):  # True表示查找所有标签
        if 'class' in tag.attrs:
            all_classes.update(tag['class'])
    all_classes = list(all_classes)  # 将集合转换为列表
    all_classes_json = [{'class': cls} for cls in all_classes]
    print(all_classes_json)
    return [(link.get_text(strip=True), link['href']) for link in links if 'href' in link.attrs and link['href'].startswith('https')]


if __name__ == '__main__':
    url = 'https://www.dygang.cc/'
    links = get_links(url)
    links_json = [{'text': text, 'url': href } for text, href in links]
    print(links_json)
